

clear all

set more off 


global estimators xtreg

*is it onset (=1) or incidence(=.)?
local cheat 1

if "`cheat'" == "." {
	local on_or_in incidence
}
if "`cheat'" == "1" {
	local on_or_in onset
}

***************************************
***************************************


	

forval t = 1995/2012 {
	forval c = 2/3 {
		foreach w in overall within {
			clear
			insheet using "output/future_neural`t'_`c'_`on_or_in'_`w'.csv"
			rename v1 p_`w'
			gen id = _n
			sort id			
			save ach, replace
			clear
			use "output/test_neural`t'_`c'_`on_or_in'"
			gen id = _n
			sort id
			merge id using ach
			drop _merge	
			sort id
			save "output/test_neural`t'_`c'_`on_or_in'_merged_`w'", replace
		}
		clear
		use "output/test_neural`t'_`c'_`on_or_in'_merged_within", replace
		
		merge id using "output/test_neural`t'_`c'_`on_or_in'_merged_overall"
		drop _merge
		save "output/test_neural`t'_`c'_`on_or_in'_merged", replace
		erase "output/test_neural`t'_`c'_`on_or_in'_merged_within.dta"
		erase "output/test_neural`t'_`c'_`on_or_in'_merged_overall.dta"
		
		
	}
}



forval c = 2/3 {
	clear
	forval t = 1995/2012 {		
		append using "output/test_neural`t'_`c'_`on_or_in'_merged"
		erase "output/test_neural`t'_`c'_`on_or_in'_merged.dta"
	}
	drop id
	save output/future_neural`c'_`on_or_in', replace
}
	





******************************************

global models 

global models overall within


program prepare_roc

	
	local which_thetas ste
	*foreach x in one two {
	foreach x in one {
	
		if "`x'" == "one" {
			local before 1
		}
		if "`x'" == "two" {
			local before 2
		}
																
		*****ll=0 is all countries, 1 only countries that ever had conflict
		forvalues ll = 0/0 {
												

			local model 0
			foreach vv in overall within {
				
				clear
				use tempx
					
				keep if conflict != `2'
				
				*rename p p_`x'_model`vv'
				drop if p_`vv' == .
				drop if  `x'_before  == .
					
				keep  p_`vv' `x'_before total_conflict conflict
				
				local model = `model' + 1 


				sort p_`vv'
					
				egen rank = rank(p_`vv') if  p_`vv' != .
							
					
				sum rank
					
				local max_rank `r(max)'
								
					
					
				if `r(N)' > 0 {
																
					
					forval cut = 1 / `max_rank' {
					
						local tpn 0
						local fpn 0
						local tpd 0
						local fpd 0
						
			
						*true positive
						 gen tp = 1 if  rank>= `cut' & rank!= . &  `x'_before == 1 &  total_conflict >= `1' & conflict!=`2' 
						*false positive						
						 gen fp = 1 if   rank>= `cut' & rank!= . &  `x'_before == 0 &  total_conflict >= `1' & conflict!=`2' 
						*wars
						 sum `x'_before if    rank!= . & total_conflict >= `1' & conflict!=`2'
						if `r(N)'!= 0 {
							*total `x'_before within evaluation sample
							local tpd = `r(N)' * `r(mean)'
							local fpd = `r(N)' * (1 - `r(mean)')
						}
			
						*warning bell rang when war
						cap sum tp if   rank!= . &  total_conflict >= `1' & conflict!=`2'
						if `r(N)'!= 0 {
							local tpn = `r(N)' * `r(mean)'							
						}
					
						*warning bell rang when no war
						 sum fp if   rank!= . &  total_conflict >= `1' & conflict!=`2'
						if `r(N)'!= 0 {
							local fpn = `r(N)' * `r(mean)'							
						}
		
					
							
							
						drop tp fp
						display "`vv'"
						post buffer (`cut') (`model') (`before') (`2')  (`tpn') (`fpn') (`tpd') (`fpd')
						*file write buffer (`cut') ("`vv'") ("`x'") (`2')  (`tpn') (`fpn') (`tpd') (`fpd')
				
							
					}
					
				}
					
				drop rank
				
									
				display "`vv'"
				
				
			}
			
		}
	}

end



local runs = 0
forval c = 3(-1)2  {

	local runs = `runs' + 1
	
	*use "chris_randomforest_one_`on_or_in'_int`c'_to15_alpha3_beta001_all_both", clear
	use output/future_neural`c'_`on_or_in', clear

	
	merge 1:1 countryid year using "complete_main_new.dta", keepusing(bdbest25 bdbest1000)
	
	
						
	if `c' == 2 {
						
		quietly gen conflict = bdbest25
							
	}
						
	if `c' == 3 {
							

		quietly gen conflict = bdbest1000
							
	}

	*drop if country == ""

	sort countryid year
	
	gen total_conflict = 0	
	
	tsset countryid year
	generate one_before=conflict==0&F1.conflict==1
	generate two_before=conflict==0&(F1.conflict==1|F2.conflict==1)
	
	if "`cheat'" == "." {
		replace one_before= 1 if conflict==1&F1.conflict==1
		replace two_before= 1 if conflict==1&(F1.conflict==1|F2.conflict==1)
	}
	
	
	drop if year < 1975	
	
	

	save tempx, replace
	
	postfile buffer cut model before conftype tpn fpn tpd fpd using output/for_roc_neural_`on_or_in'`runs', replace
	prepare_roc 0 `cheat'
	postclose buffer
	
}






erase tempx.dta


forval r = 1 / `runs' {
 
	clear
	use output/for_roc_neural_`on_or_in'`r'
	
	gen modelname = ""
	local x = 1
	foreach j in overall within {
		 replace modelname = "`j'" if model == `x'
		 local x = `x' + 1
	}
	drop model

	
	gen tpr = tpn / tpd
	gen fpr = fpn / fpd	
	
	
	foreach j in $models {
		forval y = 0 / 1 {
			local new = _N + 1
	    	set obs `new'
	    	replace tpr = `y' if _n == _N
   	 		replace fpr = `y' if _n == _N
   	 		replace modelname = "`j'" if _n == _N
   	 	}
   	}

	save output/blub, replace
	collapse (max) tpr, by(fpr modelname)
	save output/blub1, replace
	use output/blub
	collapse (min) tpr, by(fpr modelname)
	append using output/blub1
	save output/blub, replace
	collapse (max) fpr, by(tpr modelname)
	save output/blub1, replace
	use output/blub
	collapse (min) fpr, by(tpr modelname)
	append using output/blub1
	


	sort modelname fpr tpr
	
	
	
	duplicates drop
	
	
	
	
	*compute AUC
	
	
	sort modelname fpr tpr
	gen fpr_dif = fpr - fpr[_n-1] if modelname == modelname[_n-1]
	gen tpr_dif = tpr - tpr[_n-1] if modelname == modelname[_n-1]
	*gen area = tpr * fpr_dif
	gen area = 0.5 * tpr_dif * fpr_dif + tpr[_n-1] * fpr_dif if modelname == modelname[_n-1]
	by modelname: egen auc = sum(area)
	
	
	save output/for_roc_neural_`on_or_in'`r', replace
	
}



*stop

local on_or_in onset
*local on_or_in incidence

***plotting
local title_left Civil War
local title_right Armed Conflict

local left_solid_name Topics with Country Fixed Effects
local left_dash_name Only Topics

local right_solid_name Topics with Country Fixed Effects
local right_dash_name Only Topics




	foreach on_or_in in onset {
		*foreach x in xtreg xtlogit clogit {
		


			clear

			*graph of war
			use output/for_roc_neural_`on_or_in'1

			rename model what
			sort what fpr tpr
			

			graph drop _all

			sum auc if what == "overall" 
			if `r(N)' > 0 {
				local auc1 = substr(string(round(`r(max)',.01)), 1, 3)
				sum auc if what == "within" 
				local auc2 = substr(string(round(`r(max)',.01)), 1, 3)

				twoway (line tpr fpr if what == "overall" , lwidth(medthick)  lpattern(solid)  sort) ///
				(line tpr fpr if what == "within" , lwidth(medthick) lpattern(dash)  sort) ///
				 (line fpr fpr if what == "overall" , color(gs10) lpattern(dash) sort), ///
				subtitle("a) `title_left'", color(black)) ytitle(`"True Positive Rate"') xtitle(`"False Positive Rate"') legend( order(1 "`left_solid_name'" 2 "`left_dash_name'") size(small)) xlabel(,grid) ///
				plotregion(fcolor(white)) graphregion(fcolor(white)) name(fig1) aspectratio(1) ///	
				caption( "AUC: `left_solid_name' `auc1', `left_dash_name' `auc2'", size(small) color(black) ring(0)  placement(se))


				clear	
				*graph of armed conflict
				use output/for_roc_neural_`on_or_in'2

				rename model what
				sort what fpr tpr


				sum auc if what == "overall" 
				local auc1 = substr(string(round(`r(max)',.01)), 1, 3)
				sum auc if what == "within" 
				local auc2 = substr(string(round(`r(max)',.01)), 1, 3)

				twoway (line tpr fpr if what == "overall" ,  lwidth(medthick) lpattern(solid)  sort) ///
				(line tpr fpr if what == "within" ,  lwidth(medthick) lpattern(dash)  sort) ///	
				 (line fpr fpr if what == "overall" , color(gs10) lpattern(dash) sort), ///
				subtitle("b) `title_right'", color(black)) ytitle(`"True Positive Rate"') xtitle(`"False Positive Rate"') legend( order(1 "`right_solid_name'" 2 "`right_dash_name'") size(small)) xlabel(,grid) ///
				plotregion(fcolor(white)) graphregion(fcolor(white)) name(fig2) aspectratio(1) ///
				caption( "AUC: `right_solid_name' `auc1', `right_dash_name' `auc2'", size(small) color(black) ring(0)  placement(se))


				grc1leg fig1 fig2 , legendfrom(fig1) title("", color(black)  size(small)) subtitle("", color(black) size(small)) plotregion(fcolor(white)) graphregion(fcolor(white)) 

				graph export output/Fig19.eps ,replace
				*graph export figures/`k'_`j'_`x'_`on_or_in'.eps ,replace
				
			}


		}





